#!/usr/bin/python
# -*- coding:utf-8 -*-
# @author  : micah
# @time    :  
# @function: 
# @version :


import time
import requests
import pymongo
from multiprocessing import Process
from multiprocessing import JoinableQueue as Queue


class Tenxun():
    client = pymongo.MongoClient()
    collections = client['spiders10']['tenxun']

    def __init__(self):
        self.headers = {
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36'
        }
        self.url = 'https://pbaccess.video.qq.com/trpc.vector_layout.page_view.PageService/getPage?video_appid=3000010'
        # 存储需要传递的json数据
        self.data_queue = Queue()
        # 存放获取到的响应数据
        self.json_queue = Queue()
        # 存放提取之后的数据
        self.content_queue = Queue()

    def get_post_json(self):
        for i in range(1, 20):
            data = {
                "page_context": {"page_index": str(i)},
                "page_params": {"page_id": "channel_list_second_page", "page_type": "operation",
                                "channel_id": "100113",
                                "filter_params": "ifeature=-1&iarea=-1&iyear=-1&ipay=-1&sort=75", "page": str(i)},
                "page_bypass_params": {"params": {"page_id": "channel_list_second_page", "page_type": "operation",
                                                  "channel_id": "100113",
                                                  "filter_params": "ifeature=-1&iarea=-1&iyear=-1&ipay=-1&sort=75",
                                                  "page": str(i), "caller_id": "3000010", "platform_id": "2",
                                                  "data_mode": "default", "user_mode": "default"},
                                       "scene": "operation", "abtest_bypass_id": "4610e3d06ca518f3"
                                       }
            }
            self.data_queue.put(data)

    def get_data(self):
        while True:
            data = self.data_queue.get()
            self.data_queue.task_done()
            # 表单数据传递的是 字典 载荷数据传递的是json
            response = requests.get(self.url, headers=self.headers, json=data)
            # print(response.json())
            self.json_queue.put(response.json())

    def parse_data(self):
        while True:
            data = self.json_queue.get()
            self.json_queue.task_done()
            for video in data['data']['CardList'][0]['children_list']['list']['cards']:
                # print(video)
                item = {}
                item['second_title'] = video['params']['second_title'] if video['params'].get('second_title') else "空"
                item['title'] = video['params']['title']
                item['timelong'] = video['params'].get('timelong') if video['params'].get('timelong') else "空"
                item['opinion_score'] = video['params'].get('opinion_score') if video['params'].get(
                    'opinion_score') else "空"
                self.content_queue.put(item)

    def save_data(self):
        while True:
            item = self.content_queue.get()
            print(item)
            self.content_queue.task_done()
            self.collections.insert_one(item)

    def main(self):
        process_list = []
        p_get_json = Process(target=self.get_post_json)
        process_list.append(p_get_json)
        p_get = Process(target=self.get_data)
        process_list.append(p_get)
        p_parse = Process(target=self.parse_data)
        process_list.append(p_parse)
        p_save = Process(target=self.save_data)
        process_list.append(p_save)
        for p in process_list:
            p.daemon = True  # 必须要写在进程开始之前
            p.start()

        time.sleep(3)
        for q in [self.json_queue, self.data_queue, self.content_queue]:
            q.join()


if __name__ == '__main__':
    tx = Tenxun()
    tx.main()



